import random

import os

​

import torch 

import torch.nn as nn 

import torch.nn.functional as F 

import torchvision.transforms as transforms 

import torchvision.datasets as datasets 

import numpy as np 

import quadratic_weighted_kappa as qwk 

import transforms_extra 

import models 

​

import matplotlib

import matplotlib.pyplot as plt

​

import math

​

%matplotlib inline 

​

import visualization_rev1 as vis

workers = 4

batch_size = 1

mean = [0.383, 0.266, 0.190]

std = [0.291, 0.211, 0.171]

normvect = [-mean[0]/std[0], -mean[1]/std[1], -mean[2]/std[2]]

input_size = 640

input_channels = 3

# number of rotated configurations

configurations = 5

hidden_features = 64

# Data loading code                                                         

testdir = '../input/640_test'

#testdir = '../input/kk'

#testdir = '../input/ddb1_v02_01/640'

normalize = transforms.Normalize(mean=mean, std=std)                                                                                  

dataset_test = datasets.ImageFolder(testdir, transforms.Compose([                                        

    transforms.Resize(input_size),  

    #transforms_extra.Random90Rotation(),

    #transforms_extra.RandomVerticalFlip(),

    #transforms.RandomHorizontalFlip(),    

    transforms.ToTensor(),                                              

    normalize,

    transforms_extra.CenteredCircularMaskTensor(input_size, normvect),

]))

​

# load the model

resume = 'models/ret6_bn/model_best-QWKval0814.pth.tar'

print("=> loading checkpoint '{}'".format(resume))

checkpoint = torch.load(resume)

start_epoch = checkpoint['epoch']

best_prec1 = checkpoint['best_prec1']

import models

model = models.ret6_bn()

model.load_state_dict(checkpoint['state_dict'])

print("=> loaded checkpoint '{}' (epoch {})".format(resume, checkpoint['epoch']))

me = models.model_explainable(model)

#me = me.cuda()

me = me.cpu()

me.eval()

dataset = dataset_test

​

#image_nr = 0

#image_nr = int(len(dataset.imgs)*random.random())

image_nr = 9854

​

batch = torch.zeros(1, input_channels, input_size, input_size)

target = torch.zeros(1).long()

​

img, img_target = dataset.__getitem__(image_nr)

path, _ = dataset.imgs[image_nr]

file = os.path.basename(path)

​

batch[0].copy_(img)

​

if next(me.parameters()).is_cuda:

    batch = batch.cuda()

​

with torch.no_grad():

    input = torch.autograd.Variable(batch)

target[0] = img_target

input_rgb = batch.clone()

#print(input_rgb.size())

for i in range(3):

    input_rgb[0][i].mul_(std[i])

    input_rgb[0][i].add_(mean[i])

​

input_rgb[torch.lt(input_rgb, 0)] = 0.0

input_rgb[torch.gt(input_rgb, 1)] = 1.0

​

vis.plot(input_rgb)

​

#del input_rgb

#output, features, rf637, rf509_p, rf381, \

#rf253_p, rf189, rf125_p, rf93, rf61_p, rf45, \

#rf29_p, rf21, rf13_p, rf9, rf5_p, rf3

layer_vals = me.forward(input)

layer = 17

output = layer_vals[layer]

​

print("Image nr:" + str(image_nr))

print('File: ' + dataset.imgs[image_nr][0])

imagename = os.path.splitext(os.path.basename(dataset.imgs[image_nr][0]))[0]

print("Image name: " + imagename)

print("Tag")

print('Output before softmax: ')

print(output)

prob_pred_class, pred_class = output.max(1)

pred_class = pred_class[0]

print("Predicted class: " + str(pred_class))

print('Target: ')

print(target)

​

img_target = pred_class.item()

print(img_target)

layer = 16

features = layer_vals[layer]

score_features = torch.mul(me.lc[0].weight, features.expand(5,64))

​

objects = [str(i) for i in range(64)]

y_pos = np.arange(len(objects))

feat = score_features.data[img_target].tolist()

​

fig = plt.figure(figsize=(20,2))

fig.add_subplot(1, 1, 1)

​

plt.bar(y_pos, feat, align='center', alpha=0.5)

#plt.xticks(y_pos, objects)

#plt.ylabel('f')

plt.title('Features')

plt.show()

zpca = np.load("z_pca.npz")

pca = zpca['pca']

scaler_mean = zpca['zmean']

scaler_stdev = zpca['zstd']

​

feature_scaled = (features.data.squeeze().numpy() - scaler_mean)/scaler_stdev

pca_feat = np.dot(pca, feature_scaled).tolist()

​

fig = plt.figure(figsize=(20,2))

fig.add_subplot(1, 1, 1)

​

plt.bar(y_pos, pca_feat, align='center', alpha=0.5)

#plt.xticks(y_pos, objects)

#plt.ylabel('f')

plt.title('PCA of Features')

plt.show()

with torch.no_grad():

    score_accumulated_k = torch.autograd.Variable(torch.zeros(5, 1, input_size, input_size))

layer = 15

rf637 = layer_vals[layer]

averager = rf637.sum(3).sum(2).unsqueeze(2).unsqueeze(3).expand(5,64,4,4) + 1e-6 # adding a epsilon to avoid nans

score_rf637 = score_features.unsqueeze(2).unsqueeze(3).expand(5,64,4,4) * rf637.expand(5,64,4,4) / averager

vis.stats(score_rf637)

#print(score_rf637.sum(3).sum(2).sum(1) + me.lc[0].bias)

#vis.plot_scores(img_target,score_rf637)

vis.plot_scores(img_target,score_rf637.sum(1).unsqueeze(1), figsize=(50,50))

tmp = vis.map_scores_to_input_sz(score_rf637.sum(1).unsqueeze(1), 637)

vis.plot_scores(img_target,tmp, figsize=(100,100))

del tmp

layer = 14

rf509_p = layer_vals[layer]

rf509, mp_idx = F.max_pool2d(rf509_p, 2, 2, return_indices = True)  # 1x64x5x5

score_rf509, score_k637 = vis.propagate_score_through_conv2d_sz2_pad0(score_rf637, rf509, rf637, me.rf637)

#del val, rf509_p

vis.stats(score_rf509)

# sum of constant scores of the layer

score_k637 = score_k637.sum(1).unsqueeze(1)

print(score_k637.size())

vis.plot_scores(img_target,score_k637, figsize=(50,50))

# map to input space

score_kmapped = vis.map_scores_to_input_sz(score_k637, 637)

vis.plot_scores(img_target,score_kmapped, figsize=(100,100))

vis.stats(score_kmapped)

score_accumulated_k.add_(score_kmapped)

del score_k637

#vis.plot_scores(img_target,score_rf509)

vis.plot_scores(img_target,score_rf509.sum(1).unsqueeze(1), figsize=(50,50))

tmp = vis.map_scores_to_input_sz(score_rf509.sum(1).unsqueeze(1), 509)

vis.plot_scores(img_target,tmp, figsize=(100,100))

del tmp

score_mp = vis.propagate_score_through_maxpool_sz2x2_st2x2(score_rf509, mp_idx)

#del score_rf509, mp_idx

layer = 13

rf381 = layer_vals[layer]

score_rf381, score_k509 = vis.propagate_score_through_conv2d_sz3_pad1(score_mp, rf381, rf509_p, me.rf509)

#del score_mp, rf381

vis.stats(score_rf381)

vis.stats(score_k509)

score_k509 = score_k509.sum(1).unsqueeze(1)

print(score_k509.size())

vis.plot_scores(img_target, score_k509, figsize=(50,50))

# map to input space

score_kmapped = vis.map_scores_to_input_sz(score_k509, 509)

vis.plot_scores(img_target,score_kmapped, figsize=(100,100))

vis.stats(score_kmapped)

score_accumulated_k.add_(score_kmapped)

del score_k509

#vis.plot_scores(img_target, score_rf381)

vis.plot_scores(img_target,score_rf381.sum(1).unsqueeze(1), figsize=(50,50))

tmp = vis.map_scores_to_input_sz(score_rf381.sum(1).unsqueeze(1), 381)

vis.plot_scores(img_target,tmp, figsize=(100,100))

del tmp

layer = 12

rf253_p = layer_vals[layer]

rf253, mp_idx = F.max_pool2d(rf253_p, 2, 2, return_indices = True)  # 1x64x5x5

#del rf253_p

score_rf253, score_k381 = vis.propagate_score_through_conv2d_sz3_pad1(score_rf381, rf253, rf381, me.rf381)

#del score_rf381, val

vis.stats(score_rf253)

score_k381 = score_k381.sum(1).unsqueeze(1)

print(score_k381.size())

#vis.plot_scores(img_target, score_k381, figsize=(50,50))

# map to input space

score_kmapped = vis.map_scores_to_input_sz(score_k381, 381)

vis.plot_scores(img_target,score_kmapped, figsize=(100,100))

vis.stats(score_kmapped)

score_accumulated_k.add_(score_kmapped)

del score_k381

#vis.plot_scores(img_target,score_rf253)

vis.plot_scores(img_target,score_rf253.sum(1).unsqueeze(1), figsize=(50,50))

tmp = vis.map_scores_to_input_sz(score_rf253.sum(1).unsqueeze(1), 253)

vis.plot_scores(img_target,tmp, figsize=(100,100))

del tmp

score_mp = vis.propagate_score_through_maxpool_sz2x2_st2x2(score_rf253, mp_idx)

#del score_rf253, mp_idx

layer = 11

rf189 = layer_vals[layer]

score_rf189, score_k253 = vis.propagate_score_through_conv2d_sz3_pad1(score_mp, rf189, rf253_p, me.rf253)

#del score_mp, rf189

vis.stats(score_rf189)

score_k253 = score_k253.sum(1).unsqueeze(1)

#print(score_k253.size())

#vis.plot_scores(img_target, score_k253, figsize=(50,50))

​

# map to input space

score_kmapped = vis.map_scores_to_input_sz(score_k253, 253)

vis.plot_scores(img_target,score_kmapped, figsize=(100,100))

vis.stats(score_kmapped)

score_accumulated_k.add_(score_kmapped)

del score_k253

#vis.plot_scores(img_target,score_rf189)

vis.plot_scores(img_target,score_rf189.sum(1).unsqueeze(1), figsize=(50,50))

tmp = vis.map_scores_to_input_sz(score_rf189.sum(1).unsqueeze(1), 189)

vis.plot_scores(img_target,tmp, figsize=(100,100))

del tmp

layer = 10

rf125_p = layer_vals[layer]

rf125, mp_idx = F.max_pool2d(rf125_p, 2, 2, return_indices = True)  # 1x64x5x5

#del rf125_p

score_rf125, score_k189 = vis.propagate_score_through_conv2d_sz3_pad1(score_rf189, rf125, rf189, me.rf189)

#del score_rf189, 

vis.stats(score_rf125)

score_k189 = score_k189.sum(1).unsqueeze(1)

#print(score_k189.size())

#vis.plot_scores(img_target, score_k189, figsize=(50,50))

# map to input space

score_kmapped = vis.map_scores_to_input_sz(score_k189, 189)

vis.plot_scores(img_target,score_kmapped, figsize=(100,100))

vis.stats(score_kmapped)

score_accumulated_k.add_(score_kmapped)

del score_k189

#vis.plot_scores(img_target,score_rf125)

vis.plot_scores(img_target,score_rf125.sum(1).unsqueeze(1), figsize=(50,50))

tmp = vis.map_scores_to_input_sz(score_rf125.sum(1).unsqueeze(1), 125)

vis.plot_scores(img_target,tmp, figsize=(100,100))

del tmp

score_mp = vis.propagate_score_through_maxpool_sz2x2_st2x2(score_rf125, mp_idx)

#del score_rf125, mp_idx

layer = 9

rf93 = layer_vals[layer]

score_rf93, score_k125 = vis.propagate_score_through_conv2d_sz3_pad1(score_mp, rf93, rf125_p, me.rf125)

#del score_mp, rf93

vis.stats(score_rf93)

score_k125 = score_k125.sum(1).unsqueeze(1)

#print(score_k125.size())

#vis.plot_scores(img_target, score_k125, figsize=(50,50))

# map to input space

score_kmapped = vis.map_scores_to_input_sz(score_k125, 125)

vis.plot_scores(img_target,score_kmapped, figsize=(100,100))

vis.stats(score_kmapped)

score_accumulated_k.add_(score_kmapped)

del score_k125

#vis.plot_scores(img_target,score_rf93)

vis.plot_scores(img_target,score_rf93.sum(1).unsqueeze(1), figsize=(50,50))

tmp = vis.map_scores_to_input_sz(score_rf93.sum(1).unsqueeze(1), 93)

vis.plot_scores(img_target,tmp, figsize=(100,100))

del tmp

layer = 8

rf61_p = layer_vals[layer]

rf61, mp_idx = F.max_pool2d(rf61_p, 2, 2, return_indices = True)  # 1x64x5x5

score_rf61, score_k93 = vis.propagate_score_through_conv2d_sz3_pad1(score_rf93, rf61, rf93, me.rf93)

#del score_rf93, val

vis.stats(score_rf61)

score_k93 = score_k93.sum(1).unsqueeze(1)

#print(score_k93.size())

#vis.plot_scores(img_target, score_k93, figsize=(50,50))

# map to input space

score_kmapped = vis.map_scores_to_input_sz(score_k93, 93)

vis.plot_scores(img_target,score_kmapped, figsize=(100,100))

vis.stats(score_kmapped)

score_accumulated_k.add_(score_kmapped)

del score_k93

#vis.plot_scores(img_target,score_rf61)

vis.plot_scores(img_target,score_rf61.sum(1).unsqueeze(1), figsize=(50,50))

tmp = vis.map_scores_to_input_sz(score_rf61.sum(1).unsqueeze(1), 61)

vis.plot_scores(img_target,tmp, figsize=(100,100))

del tmp

score_mp = vis.propagate_score_through_maxpool_sz2x2_st2x2(score_rf61, mp_idx)

#del score_rf61, mp_idx

layer = 7

rf45 = layer_vals[layer]

score_rf45, score_k61 = vis.propagate_score_through_conv2d_sz3_pad1(score_mp, rf45, rf61_p, me.rf61)

#del score_mp, rf45

vis.stats(score_rf45)

score_k61 = score_k61.sum(1).unsqueeze(1)

print(score_k61.size())

#vis.plot_scores(img_target, score_k61, figsize=(50,50))

# map to input space

score_kmapped = vis.map_scores_to_input_sz(score_k61, 61)

vis.plot_scores(img_target,score_kmapped, figsize=(100,100))

vis.stats(score_kmapped)

score_accumulated_k.add_(score_kmapped)

del score_k61

#vis.plot_scores(img_target,score_rf45)

vis.plot_scores(img_target,score_rf45.sum(1).unsqueeze(1), figsize=(50,50))

tmp = vis.map_scores_to_input_sz(score_rf45.sum(1).unsqueeze(1), 45)

vis.plot_scores(img_target,tmp, figsize=(100,100))

del tmp

layer = 6

rf29_p = layer_vals[layer]

rf29, mp_idx = F.max_pool2d(rf29_p, 2, 2, return_indices = True)  # 1x64x5x5

#del rf29_p

score_rf29, score_k45 = vis.propagate_score_through_conv2d_sz3_pad1(score_rf45, rf29, rf45, me.rf45)

#del score_rf45, val

vis.stats(score_rf29)

score_k45 = score_k45.sum(1).unsqueeze(1)

print(score_k45.size())

#vis.plot_scores(img_target, score_k45, figsize=(50,50))

# map to input space

score_kmapped = vis.map_scores_to_input_sz(score_k45.sum(1).unsqueeze(1), 45)

vis.plot_scores(img_target,score_kmapped, figsize=(100,100))

vis.stats(score_kmapped)

score_accumulated_k.add_(score_kmapped)

del score_k45

#vis.plot_scores(img_target,score_rf29)

vis.plot_scores(img_target,score_rf29.sum(1).unsqueeze(1), figsize=(50,50), threshold=None)

tmp = vis.map_scores_to_input_sz(score_rf29.sum(1).unsqueeze(1), 29)

vis.plot_scores(img_target,tmp, figsize=(100,100))

del tmp

del score_rf637, score_rf509, score_rf381, score_rf253, score_rf189

del score_rf125, score_rf93, score_rf61, score_rf45, score_mp

del rf637, rf509, rf381, rf253, rf189, rf125, rf93, rf61, rf45

del rf509_p, rf253_p, rf125_p, rf61_p

score_mp = vis.propagate_score_through_maxpool_sz2x2_st2x2(score_rf29, mp_idx)

#del score_rf29, mp_idx

layer = 5

rf21 = layer_vals[layer]

score_rf21, score_k29 = vis.propagate_score_through_conv2d_sz3_pad1(score_mp, rf21, rf29_p, me.rf29)

#del score_mp, rf45

vis.stats(score_rf21)

score_k29 = score_k29.sum(1).unsqueeze(1)

print(score_k29.size())

#vis.plot_scores(img_target, score_k29, figsize=(50,50))

# map to input space

score_kmapped = vis.map_scores_to_input_sz(score_k29, 29)

vis.plot_scores(img_target,score_kmapped, figsize=(100,100))

vis.stats(score_kmapped)

score_accumulated_k.add_(score_kmapped)

del score_k29

#vis.plot_scores(img_target, score_rf21)

vis.plot_scores(img_target,score_rf21.sum(1).unsqueeze(1), figsize=(50,50))

tmp = vis.map_scores_to_input_sz(score_rf21.sum(1).unsqueeze(1), 21)

vis.plot_scores(img_target,tmp, figsize=(100,100))

del tmp

layer = 4

rf13_p = layer_vals[layer]

rf13, mp_idx = F.max_pool2d(rf13_p, 2, 2, return_indices = True)  # 1x64x5x5

score_rf13, score_k21 = vis.propagate_score_through_conv2d_sz3_pad1(score_rf21, rf13, rf21, me.rf21)

vis.stats(score_rf13)

score_k21 = score_k21.sum(1).unsqueeze(1)

print(score_k21.size())

#vis.plot_scores(img_target, score_k21, figsize=(50,50))

# map to input space

score_kmapped = vis.map_scores_to_input_sz(score_k21, 21)

vis.plot_scores(img_target,score_kmapped, figsize=(100,100))

vis.stats(score_kmapped)

score_accumulated_k.add_(score_kmapped)

del score_k21

#vis.plot_scores(img_target, score_rf13)

vis.plot_scores(img_target,score_rf13.sum(1).unsqueeze(1), figsize=(50,50))

tmp = vis.map_scores_to_input_sz(score_rf13.sum(1).unsqueeze(1), 13)

vis.plot_scores(img_target,tmp, figsize=(100,100))

del tmp

score_mp = vis.propagate_score_through_maxpool_sz2x2_st2x2(score_rf13, mp_idx)

#del score_rf29, mp_idx

layer = 3

rf9 = layer_vals[layer]

score_rf9, score_k13 = vis.propagate_score_through_conv2d_sz3_pad1(score_mp, rf9, rf13_p, me.rf13)

#del score_mp, rf45

vis.stats(score_rf9)

score_k13 = score_k13.sum(1).unsqueeze(1)

print(score_k13.size())

#vis.plot_scores(img_target, score_k13, figsize=(50,50))

# map to input space

score_kmapped = vis.map_scores_to_input_sz(score_k13, 13)

vis.plot_scores(img_target,score_kmapped, figsize=(100,100))

vis.stats(score_kmapped)

score_accumulated_k.add_(score_kmapped)

del score_k13

#vis.plot_scores(img_target, score_rf9)

vis.plot_scores(img_target,score_rf9.sum(1).unsqueeze(1), figsize=(50,50))

tmp = vis.map_scores_to_input_sz(score_rf9.sum(1).unsqueeze(1), 9)

vis.plot_scores(img_target,tmp, figsize=(100,100))

del tmp

layer = 2

rf5_p = layer_vals[layer]

rf5, mp_idx = F.max_pool2d(rf5_p, 2, 2, return_indices = True)  # 1x64x5x5

score_rf5, score_k9 = vis.propagate_score_through_conv2d_sz3_pad1(score_rf9, rf5, rf9, me.rf9)

vis.stats(score_rf5)

score_k9 = score_k9.sum(1).unsqueeze(1)

print(score_k9.size())

#vis.plot_scores(img_target, score_k9, figsize=(50,50))

# map to input space

score_kmapped = vis.map_scores_to_input_sz(score_k9, 9)

vis.plot_scores(img_target,score_kmapped, figsize=(100,100))

vis.stats(score_kmapped)

score_accumulated_k.add_(score_kmapped)

del score_k9

#vis.plot_scores(img_target, score_rf5)

vis.plot_scores(img_target,score_rf5.sum(1).unsqueeze(1), figsize=(50,50))

tmp = vis.map_scores_to_input_sz(score_rf5.sum(1).unsqueeze(1), 5)

vis.plot_scores(img_target,tmp, figsize=(100,100))

del tmp

del score_rf21, score_rf13, score_rf9

del rf21, rf13, rf9

del rf29_p, rf13_p

score_mp = vis.propagate_score_through_maxpool_sz2x2_st2x2(score_rf5, mp_idx)

#del score_rf29, mp_idx

layer = 1

rf3 = layer_vals[layer]

score_rf3, score_k5 = vis.propagate_score_through_conv2d_sz3_pad1(score_mp, rf3, rf5_p, me.rf5)

#del score_mp, rf4

vis.stats(score_rf3)

score_k5 = score_k5.sum(1).unsqueeze(1)

print(score_k5.size())

#vis.plot_scores(img_target, score_k5, figsize=(50,50))

# map to input space

score_kmapped = vis.map_scores_to_input_sz(score_k5, 5)

vis.plot_scores(img_target,score_kmapped, figsize=(100,100))

vis.stats(score_kmapped)

score_accumulated_k.add_(score_kmapped)

del score_k5

#vis.plot_scores(img_target, score_rf3)